import cheerio from 'cheerio'
import rp from 'request-promise'
import R from 'ramda'
import { writeFileSync } from 'fs'
import { resolve } from 'path';

const sleep = time => new Promise(resolve => setTimeout(resolve,time)) //发动一次休息

export const getarea = async () =>{
    const options={
        uri:`https://hz.fang.anjuke.com/loupan/all/`,
        transform: body => cheerio.load(body),
    }
    const $ = await rp(options)
    let area = []
    $(".filter-position .item-bd .filter").eq(0).find("a").each(function(){ //这里不能用箭头函数，会拿不到this
        const link = $(this).attr("href");
        const text = $(this).text()
        area.push({
            link,
            text
        })
    })

    //细化处理
    const fn = R.compose(
        R.map((area) =>{
            const reg = /\/loupan\/(.*)\//;
            const _id = area.link.match(reg)[1]
            area._id = _id
            return area
        }),
        R.filter(area => area.link && area.text) //判断数据是否齐全，字段不全则省去
    )

    area = fn(area);
        
    console.log("已爬到"+area.length+"条数据")
    writeFileSync("./server/database/json/Area.json",JSON.stringify(area,null,2),'utf-8')
}

const fetchAreaDetail = async (url)=>{ //获取区域下的小分区
    const options={
        uri:url,
        transform: body => cheerio.load(body),
    }

    const $ = await rp(options)
    let areaDetail = [];
    $(".filter-position .filter-sub").eq(0).find("a").each(function(){ //这里不能用箭头函数，会拿不到this
        const link = $(this).attr("href");
        const text = $(this).text()
        areaDetail.push({
            link,
            text
        })
        const fn = R.compose(
            R.map((area) =>{
                const reg = /\/loupan\/(.*)\//;
                const _id = area.link.match(reg)[1]
                area._id = _id
                return area
            }),
            R.filter(area => area.link && area.text) //判断数据是否齐全，字段不全则省去
        )
    
        areaDetail = fn(areaDetail);
    })

    return areaDetail
}



export const getAreaDetail = async () =>{
    const area = require(resolve(__dirname,'../database/json/Area.json'))
    for(let i = 0; i<area.length; i++){
        if(!area[i].areaDetail){
            const data = await fetchAreaDetail(area[i].link)
            console.log(`正在爬取${data.length}条`)
            area[i].areaDetail = data
            await sleep(500)
        }
    }

    writeFileSync("./server/database/json/AreaDetail.json",JSON.stringify(area,null,2),'utf-8')
}


